{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bi-LSTM Conditional Random Field Discussion\n",
    "- https://pytorch.org/tutorials/beginner/nlp/advanced_tutorial.html\n",
    "- https://pytorch.apachecn.org/docs/0.3/nlp_advanced_tutorial.html\n",
    "- 《Log-Linear Models, MEMMs, and CRFs》"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.autograd as autograd\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7f05500945d0>"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "def argmax(vec):\n",
    "    # 返回最大概率对应的类别\n",
    "    _, idx = torch.max(vec, 1)\n",
    "    return idx.item()\n",
    "\n",
    "\n",
    "def prepare_sequence(seq, to_ix):\n",
    "    idxs = [to_ix[w] for w in seq]\n",
    "    return torch.tensor(idxs, dtype=torch.long)\n",
    "\n",
    "\n",
    "# 使用数值上稳定的方法为前向算法计算指数和的对数\n",
    "def log_sum_exp(vec):\n",
    "    # 等于torch.log(torch.sum(torch.exp(vec)))\n",
    "    max_score = vec[0, argmax(vec)]\n",
    "    max_score_broadcast = max_score.view(1, -1).expand(1, vec.size()[1])\n",
    "    return max_score + \\\n",
    "        torch.log(torch.sum(torch.exp(vec - max_score_broadcast)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiLSTM_CRF(nn.Module):\n",
    "\n",
    "    def __init__(self, vocab_size, tag_to_ix, embedding_dim, hidden_dim):\n",
    "        super(BiLSTM_CRF, self).__init__()\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.vocab_size = vocab_size\n",
    "        self.tag_to_ix = tag_to_ix\n",
    "        self.tagset_size = len(tag_to_ix)\n",
    "\n",
    "        self.word_embeds = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.lstm = nn.LSTM(embedding_dim, hidden_dim // 2,\n",
    "                            num_layers=1, bidirectional=True)\n",
    "\n",
    "        # 将LSTM的输出映射到标记空间\n",
    "        self.hidden2tag = nn.Linear(hidden_dim, self.tagset_size)\n",
    "\n",
    "        # 过渡参数矩阵. 条目 i,j 是 *从* j *到* i 的过渡的分数\n",
    "        self.transitions = nn.Parameter(\n",
    "            torch.randn(self.tagset_size, self.tagset_size))\n",
    "        \n",
    "        print(\"self.transitions:\", self.transitions)\n",
    "        \n",
    "        # 这两句声明强制约束了我们不能向开始标记标注传递和从结束标注传递\n",
    "        self.transitions.data[tag_to_ix[START_TAG], :] = -10000\n",
    "        self.transitions.data[:, tag_to_ix[STOP_TAG]] = -10000\n",
    "        print(\"self.transitions ------>:\", self.transitions)\n",
    "        self.hidden = self.init_hidden()\n",
    "\n",
    "    def init_hidden(self):\n",
    "        return (torch.randn(2, 1, self.hidden_dim // 2),\n",
    "                torch.randn(2, 1, self.hidden_dim // 2))\n",
    "\n",
    "    def _forward_alg(self, feats):\n",
    "        # 2  feats (seq, tag_nums)\n",
    "        # 执行前向算法来计算分割函数\n",
    "        init_alphas = torch.full((1, self.tagset_size), -10000.)\n",
    "        print(\"init_alphas:\", init_alphas.size())\n",
    "        # START_TAG 包含所有的分数\n",
    "        init_alphas[0][self.tag_to_ix[START_TAG]] = 0.\n",
    "\n",
    "        # 将其包在一个变量类型中继而得到自动的反向传播\n",
    "        forward_var = init_alphas\n",
    "\n",
    "        # 在句子中迭代\n",
    "        # 第一个词的(1, score)\n",
    "        for feat in feats:\n",
    "            print(\" one feat:\", feat.size())\n",
    "            alphas_t = []  # 当前时间步的前向变量\n",
    "            print(\"-\" * 50)\n",
    "            for next_tag in range(self.tagset_size):\n",
    "                print(\"next_tag::\", next_tag)\n",
    "                # 对 emission 得分执行广播机制: 它总是相同的,\n",
    "                # 不论前一个标注如何\n",
    "                emit_score = feat[next_tag].view(1, -1).expand(1, self.tagset_size)\n",
    "                print(\"emit_score:\", emit_score, emit_score.size())\n",
    "                # trans_score 第 i 个条目是从i过渡到 next_tag 的分数（转移到当前标签）\n",
    "                trans_score = self.transitions[next_tag].view(1, -1)\n",
    "                print(\"trans_score:\", trans_score, trans_score.size())\n",
    "                # next_tag_var 第 i 个条目是在我们执行 对数-求和-指数 前\n",
    "                # 边缘的值 (i -> next_tag)\n",
    "                print(\"forward_var:\", forward_var, forward_var.size())\n",
    "                next_tag_var = forward_var + trans_score + emit_score\n",
    "                print(\"next_tag_var:\", next_tag_var, next_tag_var.size())\n",
    "                # 这个标注的前向变量是对所有的分数执行 对数-求和-指数\n",
    "                alphas_t.append(log_sum_exp(next_tag_var).view(1))\n",
    "                print(\"alphas_t:\", alphas_t, len(alphas_t))\n",
    "            forward_var = torch.cat(alphas_t).view(1, -1)\n",
    "            print(\"forward_var:\", forward_var, forward_var.size())\n",
    "        terminal_var = forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]\n",
    "        print(\"self.transitions[self.tag_to_ix[STOP_TAG]\", self.tag_to_ix[STOP_TAG], self.transitions[self.tag_to_ix[STOP_TAG]])\n",
    "        alpha = log_sum_exp(terminal_var)\n",
    "        return alpha\n",
    "\n",
    "    def _get_lstm_features(self, sentence):\n",
    "        # 1\n",
    "        self.hidden = self.init_hidden()\n",
    "        embeds = self.word_embeds(sentence).view(len(sentence), 1, -1)\n",
    "        # (batch_size, seq_len, hidden_dim//2*2)\n",
    "        lstm_out, self.hidden = self.lstm(embeds, self.hidden)\n",
    "        # 去掉中间的\n",
    "        print(\"lstm_out:\", lstm_out.size())\n",
    "        lstm_out = lstm_out.view(len(sentence), self.hidden_dim)\n",
    "        lstm_feats = self.hidden2tag(lstm_out)\n",
    "        print(\"lstm_feats:\", lstm_feats.size())\n",
    "        # (seq, tag_nums)\n",
    "        return lstm_feats\n",
    "\n",
    "    def _score_sentence(self, feats, tags):\n",
    "        # 给出标记序列的分数\n",
    "        score = torch.zeros(1)\n",
    "        tags = torch.cat([torch.tensor([self.tag_to_ix[START_TAG]], dtype=torch.long), tags])\n",
    "        for i, feat in enumerate(feats):\n",
    "            score = score + \\\n",
    "                self.transitions[tags[i + 1], tags[i]] + feat[tags[i + 1]]\n",
    "        score = score + self.transitions[self.tag_to_ix[STOP_TAG], tags[-1]]\n",
    "        return score\n",
    "\n",
    "    def _viterbi_decode(self, feats):\n",
    "        backpointers = []\n",
    "\n",
    "        # 在对数空间中初始化维特比变量\n",
    "        init_vvars = torch.full((1, self.tagset_size), -10000.)\n",
    "        init_vvars[0][self.tag_to_ix[START_TAG]] = 0\n",
    "\n",
    "        # 在第 i 步的 forward_var 存放第 i-1 步的维特比变量\n",
    "        forward_var = init_vvars\n",
    "        for feat in feats:\n",
    "            bptrs_t = []        # 存放这一步的后指针\n",
    "            viterbivars_t = []  # 存放这一步的维特比变量\n",
    "\n",
    "            for next_tag in range(self.tagset_size):\n",
    "                # next_tag_var[i] 存放先前一步标注i的\n",
    "                # 维特比变量, 加上了从标注 i 到 next_tag 的过渡的分数\n",
    "                # 我们在这里并没有将 emission 分数包含进来, 因为\n",
    "                # 最大值并不依赖于它们(我们在下面对它们进行的是相加)\n",
    "                next_tag_var = forward_var + self.transitions[next_tag]\n",
    "                best_tag_id = argmax(next_tag_var)\n",
    "                bptrs_t.append(best_tag_id)\n",
    "                viterbivars_t.append(next_tag_var[0][best_tag_id].view(1))\n",
    "            # 现在将所有 emission 得分相加, 将 forward_var\n",
    "            # 赋值到我们刚刚计算出来的维特比变量集合\n",
    "            forward_var = (torch.cat(viterbivars_t) + feat).view(1, -1)\n",
    "            backpointers.append(bptrs_t)\n",
    "\n",
    "        # 过渡到 STOP_TAG\n",
    "        terminal_var = forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]\n",
    "        best_tag_id = argmax(terminal_var)\n",
    "        path_score = terminal_var[0][best_tag_id]\n",
    "\n",
    "        # 跟着后指针去解码最佳路径\n",
    "        best_path = [best_tag_id]\n",
    "        for bptrs_t in reversed(backpointers):\n",
    "            best_tag_id = bptrs_t[best_tag_id]\n",
    "            best_path.append(best_tag_id)\n",
    "        # 弹出开始的标签 (我们并不希望把这个返回到调用函数)\n",
    "        start = best_path.pop()\n",
    "        assert start == self.tag_to_ix[START_TAG]  # 健全性检查\n",
    "        best_path.reverse()\n",
    "        return path_score, best_path\n",
    "\n",
    "    def neg_log_likelihood(self, sentence, tags):\n",
    "        #(seq_len, tag_size)\n",
    "        feats = self._get_lstm_features(sentence)\n",
    "        # 所有可能tag链分数总和\n",
    "        forward_score = self._forward_alg(feats)\n",
    "        # 真实句子分数\n",
    "        gold_score = self._score_sentence(feats, tags)\n",
    "        return forward_score - gold_score\n",
    "\n",
    "    def forward(self, sentence):  # 不要把这和上面的 _forward_alg 混淆\n",
    "        # 得到 BiLSTM 输出分数\n",
    "        lstm_feats = self._get_lstm_features(sentence)\n",
    "\n",
    "        # 给定特征, 找到最好的路径\n",
    "        score, tag_seq = self._viterbi_decode(lstm_feats)\n",
    "        return score, tag_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "START_TAG = \"<START>\"\n",
    "STOP_TAG = \"<STOP>\"\n",
    "EMBEDDING_DIM = 5\n",
    "HIDDEN_DIM = 4\n",
    "\n",
    "# 制造训练数据\n",
    "training_data = [(\n",
    "    \"长 城 位 于 北 京 市 延 庆 县\".split(),\n",
    "    \"B I O O B I I B I I\".split()\n",
    "), (\n",
    "    \"中 国 最 好 的 大 学 是 清 华 大 学\".split(),\n",
    "    \"B I O O O B I O B I I I\".split()\n",
    ")]\n",
    "\n",
    "\n",
    "word_to_ix = {}\n",
    "for sentence, tags in training_data:\n",
    "    for word in sentence:\n",
    "        if word not in word_to_ix:\n",
    "            word_to_ix[word] = len(word_to_ix)\n",
    "\n",
    "tag_to_ix = {\"B\": 0, \"I\": 1, \"O\": 2, START_TAG: 3, STOP_TAG: 4}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'长': 0, '城': 1, '位': 2, '于': 3, '北': 4, '京': 5, '市': 6, '延': 7, '庆': 8, '县': 9, '中': 10, '国': 11, '最': 12, '好': 13, '的': 14, '大': 15, '学': 16, '是': 17, '清': 18, '华': 19}\n"
     ]
    }
   ],
   "source": [
    "print(word_to_ix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "self.transitions: Parameter containing:\n",
      "tensor([[-9.1739e-01,  9.8618e-01,  4.3756e-01, -6.4480e-02,  1.1781e+00],\n",
      "        [-1.6687e+00, -2.4124e+00,  6.4837e-01,  5.7270e-01, -1.2827e+00],\n",
      "        [-1.4402e+00, -1.2760e-01, -1.8822e-01,  1.6163e+00, -1.0897e+00],\n",
      "        [-9.1565e-04,  5.6519e-01,  1.2051e+00, -6.8299e-02, -4.8749e-01],\n",
      "        [-4.9711e-01,  1.4690e+00,  3.7535e-02,  6.5826e-02, -9.1309e-01]],\n",
      "       requires_grad=True)\n",
      "self.transitions ------>: Parameter containing:\n",
      "tensor([[-9.1739e-01,  9.8618e-01,  4.3756e-01, -6.4480e-02, -1.0000e+04],\n",
      "        [-1.6687e+00, -2.4124e+00,  6.4837e-01,  5.7270e-01, -1.0000e+04],\n",
      "        [-1.4402e+00, -1.2760e-01, -1.8822e-01,  1.6163e+00, -1.0000e+04],\n",
      "        [-1.0000e+04, -1.0000e+04, -1.0000e+04, -1.0000e+04, -1.0000e+04],\n",
      "        [-4.9711e-01,  1.4690e+00,  3.7535e-02,  6.5826e-02, -1.0000e+04]],\n",
      "       requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "model = BiLSTM_CRF(len(word_to_ix), tag_to_ix, EMBEDDING_DIM, HIDDEN_DIM)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01, weight_decay=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lstm_out: torch.Size([10, 1, 4])\n",
      "lstm_feats: torch.Size([10, 5])\n",
      "训练前： ['长', '城', '位', '于', '北', '京', '市', '延', '庆', '县'] (tensor(5.0105), [2, 1, 2, 1, 2, 1, 2, 1, 2, 1])\n",
      "tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])\n",
      "lstm_out: torch.Size([10, 1, 4])\n",
      "lstm_feats: torch.Size([10, 5])\n",
      "init_alphas: torch.Size([1, 5])\n",
      " one feat: torch.Size([5])\n",
      "--------------------------------------------------\n",
      "next_tag:: 0\n",
      "emit_score: tensor([[0.4104, 0.4104, 0.4104, 0.4104, 0.4104]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-9.1739e-01,  9.8618e-01,  4.3756e-01, -6.4480e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[-10000., -10000., -10000.,      0., -10000.]]) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[-1.0001e+04, -9.9986e+03, -9.9992e+03,  3.4595e-01, -2.0000e+04]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([0.3459], grad_fn=<ViewBackward>)] 1\n",
      "next_tag:: 1\n",
      "emit_score: tensor([[0.2503, 0.2503, 0.2503, 0.2503, 0.2503]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.6687e+00, -2.4124e+00,  6.4837e-01,  5.7270e-01, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[-10000., -10000., -10000.,      0., -10000.]]) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[-1.0001e+04, -1.0002e+04, -9.9991e+03,  8.2302e-01, -2.0000e+04]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([0.3459], grad_fn=<ViewBackward>), tensor([0.8230], grad_fn=<ViewBackward>)] 2\n",
      "next_tag:: 2\n",
      "emit_score: tensor([[-0.4697, -0.4697, -0.4697, -0.4697, -0.4697]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.4402e+00, -1.2760e-01, -1.8822e-01,  1.6163e+00, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[-10000., -10000., -10000.,      0., -10000.]]) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[-1.0002e+04, -1.0001e+04, -1.0001e+04,  1.1466e+00, -2.0000e+04]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([0.3459], grad_fn=<ViewBackward>), tensor([0.8230], grad_fn=<ViewBackward>), tensor([1.1466], grad_fn=<ViewBackward>)] 3\n",
      "next_tag:: 3\n",
      "emit_score: tensor([[-0.0036, -0.0036, -0.0036, -0.0036, -0.0036]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-10000., -10000., -10000., -10000., -10000.]], grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[-10000., -10000., -10000.,      0., -10000.]]) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[-20000.0039, -20000.0039, -20000.0039, -10000.0039, -20000.0039]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([0.3459], grad_fn=<ViewBackward>), tensor([0.8230], grad_fn=<ViewBackward>), tensor([1.1466], grad_fn=<ViewBackward>), tensor([-10000.0039], grad_fn=<ViewBackward>)] 4\n",
      "next_tag:: 4\n",
      "emit_score: tensor([[-0.3023, -0.3023, -0.3023, -0.3023, -0.3023]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-4.9711e-01,  1.4690e+00,  3.7535e-02,  6.5826e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[-10000., -10000., -10000.,      0., -10000.]]) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[-1.0001e+04, -9.9988e+03, -1.0000e+04, -2.3649e-01, -2.0000e+04]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([0.3459], grad_fn=<ViewBackward>), tensor([0.8230], grad_fn=<ViewBackward>), tensor([1.1466], grad_fn=<ViewBackward>), tensor([-10000.0039], grad_fn=<ViewBackward>), tensor([-0.2365], grad_fn=<ViewBackward>)] 5\n",
      "forward_var: tensor([[ 3.4595e-01,  8.2302e-01,  1.1466e+00, -1.0000e+04, -2.3649e-01]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      " one feat: torch.Size([5])\n",
      "--------------------------------------------------\n",
      "next_tag:: 0\n",
      "emit_score: tensor([[0.3177, 0.3177, 0.3177, 0.3177, 0.3177]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-9.1739e-01,  9.8618e-01,  4.3756e-01, -6.4480e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 3.4595e-01,  8.2302e-01,  1.1466e+00, -1.0000e+04, -2.3649e-01]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[-2.5370e-01,  2.1270e+00,  1.9019e+00, -9.9998e+03, -9.9999e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([2.7640], grad_fn=<ViewBackward>)] 1\n",
      "next_tag:: 1\n",
      "emit_score: tensor([[0.2904, 0.2904, 0.2904, 0.2904, 0.2904]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.6687e+00, -2.4124e+00,  6.4837e-01,  5.7270e-01, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 3.4595e-01,  8.2302e-01,  1.1466e+00, -1.0000e+04, -2.3649e-01]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[-1.0323e+00, -1.2989e+00,  2.0854e+00, -9.9991e+03, -9.9999e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([2.7640], grad_fn=<ViewBackward>), tensor([2.1607], grad_fn=<ViewBackward>)] 2\n",
      "next_tag:: 2\n",
      "emit_score: tensor([[-0.4401, -0.4401, -0.4401, -0.4401, -0.4401]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.4402e+00, -1.2760e-01, -1.8822e-01,  1.6163e+00, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 3.4595e-01,  8.2302e-01,  1.1466e+00, -1.0000e+04, -2.3649e-01]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[-1.5343e+00,  2.5536e-01,  5.1831e-01, -9.9988e+03, -1.0001e+04]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([2.7640], grad_fn=<ViewBackward>), tensor([2.1607], grad_fn=<ViewBackward>), tensor([1.1587], grad_fn=<ViewBackward>)] 3\n",
      "next_tag:: 3\n",
      "emit_score: tensor([[0.0854, 0.0854, 0.0854, 0.0854, 0.0854]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-10000., -10000., -10000., -10000., -10000.]], grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 3.4595e-01,  8.2302e-01,  1.1466e+00, -1.0000e+04, -2.3649e-01]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ -9999.5693,  -9999.0918,  -9998.7686, -19999.9180, -10000.1514]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([2.7640], grad_fn=<ViewBackward>), tensor([2.1607], grad_fn=<ViewBackward>), tensor([1.1587], grad_fn=<ViewBackward>), tensor([-9997.8828], grad_fn=<ViewBackward>)] 4\n",
      "next_tag:: 4\n",
      "emit_score: tensor([[-0.2287, -0.2287, -0.2287, -0.2287, -0.2287]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-4.9711e-01,  1.4690e+00,  3.7535e-02,  6.5826e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 3.4595e-01,  8.2302e-01,  1.1466e+00, -1.0000e+04, -2.3649e-01]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[-3.7982e-01,  2.0634e+00,  9.5547e-01, -1.0000e+04, -1.0000e+04]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([2.7640], grad_fn=<ViewBackward>), tensor([2.1607], grad_fn=<ViewBackward>), tensor([1.1587], grad_fn=<ViewBackward>), tensor([-9997.8828], grad_fn=<ViewBackward>), tensor([2.4120], grad_fn=<ViewBackward>)] 5\n",
      "forward_var: tensor([[ 2.7640e+00,  2.1607e+00,  1.1587e+00, -9.9979e+03,  2.4120e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      " one feat: torch.Size([5])\n",
      "--------------------------------------------------\n",
      "next_tag:: 0\n",
      "emit_score: tensor([[0.1949, 0.1949, 0.1949, 0.1949, 0.1949]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-9.1739e-01,  9.8618e-01,  4.3756e-01, -6.4480e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 2.7640e+00,  2.1607e+00,  1.1587e+00, -9.9979e+03,  2.4120e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 2.0416e+00,  3.3418e+00,  1.7911e+00, -9.9978e+03, -9.9974e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([3.7369], grad_fn=<ViewBackward>)] 1\n",
      "next_tag:: 1\n",
      "emit_score: tensor([[0.2991, 0.2991, 0.2991, 0.2991, 0.2991]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.6687e+00, -2.4124e+00,  6.4837e-01,  5.7270e-01, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 2.7640e+00,  2.1607e+00,  1.1587e+00, -9.9979e+03,  2.4120e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 1.3945e+00,  4.7397e-02,  2.1061e+00, -9.9970e+03, -9.9973e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([3.7369], grad_fn=<ViewBackward>), tensor([2.5876], grad_fn=<ViewBackward>)] 2\n",
      "next_tag:: 2\n",
      "emit_score: tensor([[-0.3622, -0.3622, -0.3622, -0.3622, -0.3622]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.4402e+00, -1.2760e-01, -1.8822e-01,  1.6163e+00, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 2.7640e+00,  2.1607e+00,  1.1587e+00, -9.9979e+03,  2.4120e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 9.6160e-01,  1.6708e+00,  6.0821e-01, -9.9966e+03, -9.9980e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([3.7369], grad_fn=<ViewBackward>), tensor([2.5876], grad_fn=<ViewBackward>), tensor([2.2793], grad_fn=<ViewBackward>)] 3\n",
      "next_tag:: 3\n",
      "emit_score: tensor([[0.1018, 0.1018, 0.1018, 0.1018, 0.1018]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-10000., -10000., -10000., -10000., -10000.]], grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 2.7640e+00,  2.1607e+00,  1.1587e+00, -9.9979e+03,  2.4120e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ -9997.1348,  -9997.7373,  -9998.7402, -19997.7812,  -9997.4863]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([3.7369], grad_fn=<ViewBackward>), tensor([2.5876], grad_fn=<ViewBackward>), tensor([2.2793], grad_fn=<ViewBackward>), tensor([-9996.2383], grad_fn=<ViewBackward>)] 4\n",
      "next_tag:: 4\n",
      "emit_score: tensor([[-0.3620, -0.3620, -0.3620, -0.3620, -0.3620]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-4.9711e-01,  1.4690e+00,  3.7535e-02,  6.5826e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 2.7640e+00,  2.1607e+00,  1.1587e+00, -9.9979e+03,  2.4120e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 1.9049e+00,  3.2677e+00,  8.3419e-01, -9.9982e+03, -9.9980e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([3.7369], grad_fn=<ViewBackward>), tensor([2.5876], grad_fn=<ViewBackward>), tensor([2.2793], grad_fn=<ViewBackward>), tensor([-9996.2383], grad_fn=<ViewBackward>), tensor([3.5631], grad_fn=<ViewBackward>)] 5\n",
      "forward_var: tensor([[ 3.7369e+00,  2.5876e+00,  2.2793e+00, -9.9962e+03,  3.5631e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      " one feat: torch.Size([5])\n",
      "--------------------------------------------------\n",
      "next_tag:: 0\n",
      "emit_score: tensor([[0.1523, 0.1523, 0.1523, 0.1523, 0.1523]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-9.1739e-01,  9.8618e-01,  4.3756e-01, -6.4480e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 3.7369e+00,  2.5876e+00,  2.2793e+00, -9.9962e+03,  3.5631e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 2.9718e+00,  3.7261e+00,  2.8692e+00, -9.9962e+03, -9.9963e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([4.3652], grad_fn=<ViewBackward>)] 1\n",
      "next_tag:: 1\n",
      "emit_score: tensor([[0.2306, 0.2306, 0.2306, 0.2306, 0.2306]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.6687e+00, -2.4124e+00,  6.4837e-01,  5.7270e-01, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 3.7369e+00,  2.5876e+00,  2.2793e+00, -9.9962e+03,  3.5631e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 2.2988e+00,  4.0584e-01,  3.1583e+00, -9.9954e+03, -9.9962e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([4.3652], grad_fn=<ViewBackward>), tensor([3.5551], grad_fn=<ViewBackward>)] 2\n",
      "next_tag:: 2\n",
      "emit_score: tensor([[-0.3525, -0.3525, -0.3525, -0.3525, -0.3525]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.4402e+00, -1.2760e-01, -1.8822e-01,  1.6163e+00, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 3.7369e+00,  2.5876e+00,  2.2793e+00, -9.9962e+03,  3.5631e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 1.9442e+00,  2.1075e+00,  1.7386e+00, -9.9950e+03, -9.9968e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([4.3652], grad_fn=<ViewBackward>), tensor([3.5551], grad_fn=<ViewBackward>), tensor([3.0400], grad_fn=<ViewBackward>)] 3\n",
      "next_tag:: 3\n",
      "emit_score: tensor([[0.0509, 0.0509, 0.0509, 0.0509, 0.0509]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-10000., -10000., -10000., -10000., -10000.]], grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 3.7369e+00,  2.5876e+00,  2.2793e+00, -9.9962e+03,  3.5631e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ -9996.2119,  -9997.3613,  -9997.6699, -19996.1875,  -9996.3857]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([4.3652], grad_fn=<ViewBackward>), tensor([3.5551], grad_fn=<ViewBackward>), tensor([3.0400], grad_fn=<ViewBackward>), tensor([-9995.3408], grad_fn=<ViewBackward>)] 4\n",
      "next_tag:: 4\n",
      "emit_score: tensor([[-0.4251, -0.4251, -0.4251, -0.4251, -0.4251]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-4.9711e-01,  1.4690e+00,  3.7535e-02,  6.5826e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 3.7369e+00,  2.5876e+00,  2.2793e+00, -9.9962e+03,  3.5631e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 2.8147e+00,  3.6315e+00,  1.8917e+00, -9.9966e+03, -9.9969e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([4.3652], grad_fn=<ViewBackward>), tensor([3.5551], grad_fn=<ViewBackward>), tensor([3.0400], grad_fn=<ViewBackward>), tensor([-9995.3408], grad_fn=<ViewBackward>), tensor([4.1123], grad_fn=<ViewBackward>)] 5\n",
      "forward_var: tensor([[ 4.3652e+00,  3.5551e+00,  3.0400e+00, -9.9953e+03,  4.1123e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      " one feat: torch.Size([5])\n",
      "--------------------------------------------------\n",
      "next_tag:: 0\n",
      "emit_score: tensor([[0.0359, 0.0359, 0.0359, 0.0359, 0.0359]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-9.1739e-01,  9.8618e-01,  4.3756e-01, -6.4480e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 4.3652e+00,  3.5551e+00,  3.0400e+00, -9.9953e+03,  4.1123e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 3.4838e+00,  4.5772e+00,  3.5135e+00, -9.9954e+03, -9.9959e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([5.0962], grad_fn=<ViewBackward>)] 1\n",
      "next_tag:: 1\n",
      "emit_score: tensor([[0.2854, 0.2854, 0.2854, 0.2854, 0.2854]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.6687e+00, -2.4124e+00,  6.4837e-01,  5.7270e-01, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 4.3652e+00,  3.5551e+00,  3.0400e+00, -9.9953e+03,  4.1123e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 2.9820e+00,  1.4282e+00,  3.9738e+00, -9.9945e+03, -9.9956e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([5.0962], grad_fn=<ViewBackward>), tensor([4.3449], grad_fn=<ViewBackward>)] 2\n",
      "next_tag:: 2\n",
      "emit_score: tensor([[-0.4203, -0.4203, -0.4203, -0.4203, -0.4203]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.4402e+00, -1.2760e-01, -1.8822e-01,  1.6163e+00, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 4.3652e+00,  3.5551e+00,  3.0400e+00, -9.9953e+03,  4.1123e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 2.5047e+00,  3.0073e+00,  2.4315e+00, -9.9941e+03, -9.9963e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([5.0962], grad_fn=<ViewBackward>), tensor([4.3449], grad_fn=<ViewBackward>), tensor([3.7807], grad_fn=<ViewBackward>)] 3\n",
      "next_tag:: 3\n",
      "emit_score: tensor([[0.1320, 0.1320, 0.1320, 0.1320, 0.1320]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-10000., -10000., -10000., -10000., -10000.]], grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 4.3652e+00,  3.5551e+00,  3.0400e+00, -9.9953e+03,  4.1123e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ -9995.5029,  -9996.3135,  -9996.8281, -19995.2070,  -9995.7559]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([5.0962], grad_fn=<ViewBackward>), tensor([4.3449], grad_fn=<ViewBackward>), tensor([3.7807], grad_fn=<ViewBackward>), tensor([-9994.5918], grad_fn=<ViewBackward>)] 4\n",
      "next_tag:: 4\n",
      "emit_score: tensor([[-0.4417, -0.4417, -0.4417, -0.4417, -0.4417]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-4.9711e-01,  1.4690e+00,  3.7535e-02,  6.5826e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 4.3652e+00,  3.5551e+00,  3.0400e+00, -9.9953e+03,  4.1123e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 3.4264e+00,  4.5824e+00,  2.6358e+00, -9.9957e+03, -9.9963e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([5.0962], grad_fn=<ViewBackward>), tensor([4.3449], grad_fn=<ViewBackward>), tensor([3.7807], grad_fn=<ViewBackward>), tensor([-9994.5918], grad_fn=<ViewBackward>), tensor([4.9592], grad_fn=<ViewBackward>)] 5\n",
      "forward_var: tensor([[ 5.0962e+00,  4.3449e+00,  3.7807e+00, -9.9946e+03,  4.9592e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      " one feat: torch.Size([5])\n",
      "--------------------------------------------------\n",
      "next_tag:: 0\n",
      "emit_score: tensor([[0.0857, 0.0857, 0.0857, 0.0857, 0.0857]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-9.1739e-01,  9.8618e-01,  4.3756e-01, -6.4480e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 5.0962e+00,  4.3449e+00,  3.7807e+00, -9.9946e+03,  4.9592e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 4.2645e+00,  5.4168e+00,  4.3040e+00, -9.9946e+03, -9.9950e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([5.9143], grad_fn=<ViewBackward>)] 1\n",
      "next_tag:: 1\n",
      "emit_score: tensor([[0.2034, 0.2034, 0.2034, 0.2034, 0.2034]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.6687e+00, -2.4124e+00,  6.4837e-01,  5.7270e-01, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 5.0962e+00,  4.3449e+00,  3.7807e+00, -9.9946e+03,  4.9592e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 3.6309e+00,  2.1359e+00,  4.6325e+00, -9.9938e+03, -9.9948e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([5.9143], grad_fn=<ViewBackward>), tensor([5.0038], grad_fn=<ViewBackward>)] 2\n",
      "next_tag:: 2\n",
      "emit_score: tensor([[-0.4373, -0.4373, -0.4373, -0.4373, -0.4373]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.4402e+00, -1.2760e-01, -1.8822e-01,  1.6163e+00, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 5.0962e+00,  4.3449e+00,  3.7807e+00, -9.9946e+03,  4.9592e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 3.2187e+00,  3.7800e+00,  3.1552e+00, -9.9934e+03, -9.9955e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([5.9143], grad_fn=<ViewBackward>), tensor([5.0038], grad_fn=<ViewBackward>), tensor([4.5247], grad_fn=<ViewBackward>)] 3\n",
      "next_tag:: 3\n",
      "emit_score: tensor([[0.0575, 0.0575, 0.0575, 0.0575, 0.0575]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-10000., -10000., -10000., -10000., -10000.]], grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 5.0962e+00,  4.3449e+00,  3.7807e+00, -9.9946e+03,  4.9592e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ -9994.8467,  -9995.5977,  -9996.1621, -19994.5352,  -9994.9834]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([5.9143], grad_fn=<ViewBackward>), tensor([5.0038], grad_fn=<ViewBackward>), tensor([4.5247], grad_fn=<ViewBackward>), tensor([-9993.8867], grad_fn=<ViewBackward>)] 4\n",
      "next_tag:: 4\n",
      "emit_score: tensor([[-0.4256, -0.4256, -0.4256, -0.4256, -0.4256]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-4.9711e-01,  1.4690e+00,  3.7535e-02,  6.5826e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 5.0962e+00,  4.3449e+00,  3.7807e+00, -9.9946e+03,  4.9592e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 4.1734e+00,  5.3883e+00,  3.3926e+00, -9.9950e+03, -9.9955e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([5.9143], grad_fn=<ViewBackward>), tensor([5.0038], grad_fn=<ViewBackward>), tensor([4.5247], grad_fn=<ViewBackward>), tensor([-9993.8867], grad_fn=<ViewBackward>), tensor([5.7478], grad_fn=<ViewBackward>)] 5\n",
      "forward_var: tensor([[ 5.9143e+00,  5.0038e+00,  4.5247e+00, -9.9939e+03,  5.7478e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      " one feat: torch.Size([5])\n",
      "--------------------------------------------------\n",
      "next_tag:: 0\n",
      "emit_score: tensor([[0.0251, 0.0251, 0.0251, 0.0251, 0.0251]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-9.1739e-01,  9.8618e-01,  4.3756e-01, -6.4480e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 5.9143e+00,  5.0038e+00,  4.5247e+00, -9.9939e+03,  5.7478e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 5.0220e+00,  6.0151e+00,  4.9874e+00, -9.9939e+03, -9.9942e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([6.5622], grad_fn=<ViewBackward>)] 1\n",
      "next_tag:: 1\n",
      "emit_score: tensor([[0.2328, 0.2328, 0.2328, 0.2328, 0.2328]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.6687e+00, -2.4124e+00,  6.4837e-01,  5.7270e-01, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 5.9143e+00,  5.0038e+00,  4.5247e+00, -9.9939e+03,  5.7478e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 4.4784e+00,  2.8243e+00,  5.4060e+00, -9.9931e+03, -9.9940e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([6.5622], grad_fn=<ViewBackward>), tensor([5.7920], grad_fn=<ViewBackward>)] 2\n",
      "next_tag:: 2\n",
      "emit_score: tensor([[-0.4197, -0.4197, -0.4197, -0.4197, -0.4197]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.4402e+00, -1.2760e-01, -1.8822e-01,  1.6163e+00, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 5.9143e+00,  5.0038e+00,  4.5247e+00, -9.9939e+03,  5.7478e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 4.0544e+00,  4.4565e+00,  3.9168e+00, -9.9927e+03, -9.9947e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([6.5622], grad_fn=<ViewBackward>), tensor([5.7920], grad_fn=<ViewBackward>), tensor([5.2683], grad_fn=<ViewBackward>)] 3\n",
      "next_tag:: 3\n",
      "emit_score: tensor([[0.1857, 0.1857, 0.1857, 0.1857, 0.1857]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-10000., -10000., -10000., -10000., -10000.]], grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 5.9143e+00,  5.0038e+00,  4.5247e+00, -9.9939e+03,  5.7478e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ -9993.9004,  -9994.8105,  -9995.2900, -19993.7012,  -9994.0664]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([6.5622], grad_fn=<ViewBackward>), tensor([5.7920], grad_fn=<ViewBackward>), tensor([5.2683], grad_fn=<ViewBackward>), tensor([-9992.9844], grad_fn=<ViewBackward>)] 4\n",
      "next_tag:: 4\n",
      "emit_score: tensor([[-0.1530, -0.1530, -0.1530, -0.1530, -0.1530]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-4.9711e-01,  1.4690e+00,  3.7535e-02,  6.5826e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 5.9143e+00,  5.0038e+00,  4.5247e+00, -9.9939e+03,  5.7478e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 5.2642e+00,  6.3199e+00,  4.4093e+00, -9.9940e+03, -9.9944e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([6.5622], grad_fn=<ViewBackward>), tensor([5.7920], grad_fn=<ViewBackward>), tensor([5.2683], grad_fn=<ViewBackward>), tensor([-9992.9844], grad_fn=<ViewBackward>), tensor([6.7226], grad_fn=<ViewBackward>)] 5\n",
      "forward_var: tensor([[ 6.5622e+00,  5.7920e+00,  5.2683e+00, -9.9930e+03,  6.7226e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      " one feat: torch.Size([5])\n",
      "--------------------------------------------------\n",
      "next_tag:: 0\n",
      "emit_score: tensor([[0.4723, 0.4723, 0.4723, 0.4723, 0.4723]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-9.1739e-01,  9.8618e-01,  4.3756e-01, -6.4480e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 6.5622e+00,  5.7920e+00,  5.2683e+00, -9.9930e+03,  6.7226e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 6.1171e+00,  7.2505e+00,  6.1781e+00, -9.9926e+03, -9.9928e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([7.7598], grad_fn=<ViewBackward>)] 1\n",
      "next_tag:: 1\n",
      "emit_score: tensor([[0.2881, 0.2881, 0.2881, 0.2881, 0.2881]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.6687e+00, -2.4124e+00,  6.4837e-01,  5.7270e-01, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 6.5622e+00,  5.7920e+00,  5.2683e+00, -9.9930e+03,  6.7226e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 5.1816e+00,  3.6678e+00,  6.2048e+00, -9.9921e+03, -9.9930e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([7.7598], grad_fn=<ViewBackward>), tensor([6.5684], grad_fn=<ViewBackward>)] 2\n",
      "next_tag:: 2\n",
      "emit_score: tensor([[-0.5325, -0.5325, -0.5325, -0.5325, -0.5325]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.4402e+00, -1.2760e-01, -1.8822e-01,  1.6163e+00, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 6.5622e+00,  5.7920e+00,  5.2683e+00, -9.9930e+03,  6.7226e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 4.5895e+00,  5.1319e+00,  4.5475e+00, -9.9919e+03, -9.9938e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([7.7598], grad_fn=<ViewBackward>), tensor([6.5684], grad_fn=<ViewBackward>), tensor([5.8922], grad_fn=<ViewBackward>)] 3\n",
      "next_tag:: 3\n",
      "emit_score: tensor([[5.0604e-05, 5.0604e-05, 5.0604e-05, 5.0604e-05, 5.0604e-05]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-10000., -10000., -10000., -10000., -10000.]], grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 6.5622e+00,  5.7920e+00,  5.2683e+00, -9.9930e+03,  6.7226e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ -9993.4375,  -9994.2080,  -9994.7314, -19992.9844,  -9993.2773]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([7.7598], grad_fn=<ViewBackward>), tensor([6.5684], grad_fn=<ViewBackward>), tensor([5.8922], grad_fn=<ViewBackward>), tensor([-9992.3691], grad_fn=<ViewBackward>)] 4\n",
      "next_tag:: 4\n",
      "emit_score: tensor([[-0.2954, -0.2954, -0.2954, -0.2954, -0.2954]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-4.9711e-01,  1.4690e+00,  3.7535e-02,  6.5826e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 6.5622e+00,  5.7920e+00,  5.2683e+00, -9.9930e+03,  6.7226e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 5.7697e+00,  6.9656e+00,  5.0104e+00, -9.9932e+03, -9.9936e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([7.7598], grad_fn=<ViewBackward>), tensor([6.5684], grad_fn=<ViewBackward>), tensor([5.8922], grad_fn=<ViewBackward>), tensor([-9992.3691], grad_fn=<ViewBackward>), tensor([7.3330], grad_fn=<ViewBackward>)] 5\n",
      "forward_var: tensor([[ 7.7598e+00,  6.5684e+00,  5.8922e+00, -9.9924e+03,  7.3330e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      " one feat: torch.Size([5])\n",
      "--------------------------------------------------\n",
      "next_tag:: 0\n",
      "emit_score: tensor([[0.4129, 0.4129, 0.4129, 0.4129, 0.4129]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-9.1739e-01,  9.8618e-01,  4.3756e-01, -6.4480e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 7.7598e+00,  6.5684e+00,  5.8922e+00, -9.9924e+03,  7.3330e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 7.2553e+00,  7.9676e+00,  6.7427e+00, -9.9920e+03, -9.9923e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([8.5466], grad_fn=<ViewBackward>)] 1\n",
      "next_tag:: 1\n",
      "emit_score: tensor([[0.2848, 0.2848, 0.2848, 0.2848, 0.2848]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.6687e+00, -2.4124e+00,  6.4837e-01,  5.7270e-01, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 7.7598e+00,  6.5684e+00,  5.8922e+00, -9.9924e+03,  7.3330e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 6.3759e+00,  4.4408e+00,  6.8253e+00, -9.9915e+03, -9.9924e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([8.5466], grad_fn=<ViewBackward>), tensor([7.3735], grad_fn=<ViewBackward>)] 2\n",
      "next_tag:: 2\n",
      "emit_score: tensor([[-0.4398, -0.4398, -0.4398, -0.4398, -0.4398]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.4402e+00, -1.2760e-01, -1.8822e-01,  1.6163e+00, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 7.7598e+00,  6.5684e+00,  5.8922e+00, -9.9924e+03,  7.3330e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 5.8798e+00,  6.0010e+00,  5.2641e+00, -9.9912e+03, -9.9931e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([8.5466], grad_fn=<ViewBackward>), tensor([7.3735], grad_fn=<ViewBackward>), tensor([6.8616], grad_fn=<ViewBackward>)] 3\n",
      "next_tag:: 3\n",
      "emit_score: tensor([[0.0043, 0.0043, 0.0043, 0.0043, 0.0043]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-10000., -10000., -10000., -10000., -10000.]], grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 7.7598e+00,  6.5684e+00,  5.8922e+00, -9.9924e+03,  7.3330e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ -9992.2363,  -9993.4277,  -9994.1035, -19992.3652,  -9992.6631]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([8.5466], grad_fn=<ViewBackward>), tensor([7.3735], grad_fn=<ViewBackward>), tensor([6.8616], grad_fn=<ViewBackward>), tensor([-9991.4893], grad_fn=<ViewBackward>)] 4\n",
      "next_tag:: 4\n",
      "emit_score: tensor([[-0.3557, -0.3557, -0.3557, -0.3557, -0.3557]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-4.9711e-01,  1.4690e+00,  3.7535e-02,  6.5826e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 7.7598e+00,  6.5684e+00,  5.8922e+00, -9.9924e+03,  7.3330e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 6.9069e+00,  7.6817e+00,  5.5740e+00, -9.9927e+03, -9.9930e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([8.5466], grad_fn=<ViewBackward>), tensor([7.3735], grad_fn=<ViewBackward>), tensor([6.8616], grad_fn=<ViewBackward>), tensor([-9991.4893], grad_fn=<ViewBackward>), tensor([8.1406], grad_fn=<ViewBackward>)] 5\n",
      "forward_var: tensor([[ 8.5466e+00,  7.3735e+00,  6.8616e+00, -9.9915e+03,  8.1406e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      " one feat: torch.Size([5])\n",
      "--------------------------------------------------\n",
      "next_tag:: 0\n",
      "emit_score: tensor([[0.0517, 0.0517, 0.0517, 0.0517, 0.0517]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-9.1739e-01,  9.8618e-01,  4.3756e-01, -6.4480e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 8.5466e+00,  7.3735e+00,  6.8616e+00, -9.9915e+03,  8.1406e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 7.6809e+00,  8.4114e+00,  7.3508e+00, -9.9915e+03, -9.9918e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([9.0146], grad_fn=<ViewBackward>)] 1\n",
      "next_tag:: 1\n",
      "emit_score: tensor([[0.3364, 0.3364, 0.3364, 0.3364, 0.3364]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.6687e+00, -2.4124e+00,  6.4837e-01,  5.7270e-01, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 8.5466e+00,  7.3735e+00,  6.8616e+00, -9.9915e+03,  8.1406e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 7.2143e+00,  5.2976e+00,  7.8464e+00, -9.9906e+03, -9.9915e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([9.0146], grad_fn=<ViewBackward>), tensor([8.3224], grad_fn=<ViewBackward>)] 2\n",
      "next_tag:: 2\n",
      "emit_score: tensor([[-0.4075, -0.4075, -0.4075, -0.4075, -0.4075]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-1.4402e+00, -1.2760e-01, -1.8822e-01,  1.6163e+00, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 8.5466e+00,  7.3735e+00,  6.8616e+00, -9.9915e+03,  8.1406e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 6.6989e+00,  6.8384e+00,  6.2658e+00, -9.9903e+03, -9.9923e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([9.0146], grad_fn=<ViewBackward>), tensor([8.3224], grad_fn=<ViewBackward>), tensor([7.7279], grad_fn=<ViewBackward>)] 3\n",
      "next_tag:: 3\n",
      "emit_score: tensor([[0.1421, 0.1421, 0.1421, 0.1421, 0.1421]], grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-10000., -10000., -10000., -10000., -10000.]], grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 8.5466e+00,  7.3735e+00,  6.8616e+00, -9.9915e+03,  8.1406e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ -9991.3115,  -9992.4854,  -9992.9971, -19991.3457,  -9991.7178]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([9.0146], grad_fn=<ViewBackward>), tensor([8.3224], grad_fn=<ViewBackward>), tensor([7.7279], grad_fn=<ViewBackward>), tensor([-9990.5410], grad_fn=<ViewBackward>)] 4\n",
      "next_tag:: 4\n",
      "emit_score: tensor([[-0.5091, -0.5091, -0.5091, -0.5091, -0.5091]],\n",
      "       grad_fn=<ExpandBackward>) torch.Size([1, 5])\n",
      "trans_score: tensor([[-4.9711e-01,  1.4690e+00,  3.7535e-02,  6.5826e-02, -1.0000e+04]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "forward_var: tensor([[ 8.5466e+00,  7.3735e+00,  6.8616e+00, -9.9915e+03,  8.1406e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "next_tag_var: tensor([[ 7.5404e+00,  8.3334e+00,  6.3899e+00, -9.9919e+03, -9.9924e+03]],\n",
      "       grad_fn=<AddBackward0>) torch.Size([1, 5])\n",
      "alphas_t: [tensor([9.0146], grad_fn=<ViewBackward>), tensor([8.3224], grad_fn=<ViewBackward>), tensor([7.7279], grad_fn=<ViewBackward>), tensor([-9990.5410], grad_fn=<ViewBackward>), tensor([8.8007], grad_fn=<ViewBackward>)] 5\n",
      "forward_var: tensor([[ 9.0146e+00,  8.3224e+00,  7.7279e+00, -9.9905e+03,  8.8007e+00]],\n",
      "       grad_fn=<ViewBackward>) torch.Size([1, 5])\n",
      "self.transitions[self.tag_to_ix[STOP_TAG] 4 tensor([-4.9711e-01,  1.4690e+00,  3.7535e-02,  6.5826e-02, -1.0000e+04],\n",
      "       grad_fn=<SelectBackward>)\n",
      "loss: tensor([15.9027], grad_fn=<SubBackward0>)\n",
      "lstm_out: torch.Size([10, 1, 4])\n",
      "lstm_feats: torch.Size([10, 5])\n",
      "训练后： ['长', '城', '位', '于', '北', '京', '市', '延', '庆', '县'] (tensor(5.0174), [2, 1, 2, 1, 2, 1, 2, 1, 2, 1])\n"
     ]
    }
   ],
   "source": [
    "# 训练前检查预测结果\n",
    "with torch.no_grad():\n",
    "    precheck_sent = prepare_sequence(training_data[0][0], word_to_ix)\n",
    "    precheck_tags = torch.tensor([tag_to_ix[t] for t in training_data[0][1]], dtype=torch.long)\n",
    "    print(\"训练前：\", training_data[0][0], model(precheck_sent))\n",
    "\n",
    "\n",
    "# 通常不会训这么多epochs， 这是demo数据\n",
    "# for epoch in range(300):  \n",
    "for sentence, tags in training_data:\n",
    "    # 第一步: 需要记住的是Pytorch会累积梯度\n",
    "    # 我们需要在每次实例之前把它们清除\n",
    "    # ['the', 'wall', 'street', 'journal', 'reported', 'today', 'that', 'apple', 'corporation', 'made', 'money'] \n",
    "    # ['B', 'I', 'I', 'I', 'O', 'O', 'O', 'B', 'I', 'O', 'O']\n",
    "    model.zero_grad()\n",
    "\n",
    "    # 第二步: 为我们的网络准备好输入, 即把它们转变成单词索引变量 (Variables)\n",
    "    sentence_in = prepare_sequence(sentence, word_to_ix)\n",
    "    targets = torch.tensor([tag_to_ix[t] for t in tags], dtype=torch.long)\n",
    "    print(sentence_in)\n",
    "    # 第三步: 运行前向传递\n",
    "    # 负对数似然\n",
    "    loss = model.neg_log_likelihood(sentence_in, targets)\n",
    "    print(\"loss:\", loss)\n",
    "    break\n",
    "    # 第四步: 计算损失, 梯度以及使用 optimizer.step() 来更新参数\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "# 在训练之后检查预测结果\n",
    "with torch.no_grad():\n",
    "    precheck_sent = prepare_sequence(training_data[0][0], word_to_ix)\n",
    "    print(\"训练后：\", training_data[0][0], model(precheck_sent))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.1903, 0.9510, 0.7418, 0.6104, 0.4461])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat = torch.rand(5)\n",
    "feat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.7418, 0.7418, 0.7418, 0.7418, 0.7418]])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat[2].view(1, -1).expand(1, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "expected Tensor as element 0 in argument 0, but got int",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-47-b7e543b238ee>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m: expected Tensor as element 0 in argument 0, but got int"
     ]
    }
   ],
   "source": [
    "# 使用数值上稳定的方法为前向算法计算指数和的对数\n",
    "\n",
    "def log_sum_exp(vec):\n",
    "    max_score = vec[0, argmax(vec)]\n",
    "    max_score_broadcast = max_score.view(1, -1).expand(1, vec.size()[1])\n",
    "    return max_score + \\\n",
    "        torch.log(torch.sum(torch.exp(vec - max_score_broadcast)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.9307, 0.8109, 0.4847, 0.2767, 0.5331]])"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec = torch.rand(1,5)\n",
    "vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.9307)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_score = vec[0, argmax(vec)]\n",
    "max_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.9307, 0.9307, 0.9307, 0.9307, 0.9307]])"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_score_broadcast  = max_score.view(1, -1).expand(1, vec.size()[1])\n",
    "max_score_broadcast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0000, -0.1198, -0.4460, -0.6540, -0.3975]])"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec - max_score_broadcast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1.3135)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_ = torch.log(torch.sum(torch.exp(vec - max_score_broadcast)))\n",
    "x_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.2442)"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_score + x_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.2442)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.log(torch.sum(torch.exp(vec)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.8894, 0.8167, 0.2974, 0.3956, 0.6985]])"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_t = torch.rand(1,5)\n",
    "x_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.1582) tensor(2.1582)\n"
     ]
    }
   ],
   "source": [
    "print(log_sum_exp(x_t), torch.log(torch.sum(torch.exp(x_t))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2.1000, 2.2000]])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cat([torch.tensor(2.1).view(1), torch.tensor(2.2).view(1)]).view(1, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, y = torch.max(x, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1, 1, 0],\n",
       "        [2, 0, 0, 1]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "The expanded size of the tensor (3) must match the existing size (32) at non-singleton dimension 1.  Target sizes: [1, 3].  Tensor sizes: [1, 32]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-30-7e48ce564efd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m: The expanded size of the tensor (3) must match the existing size (32) at non-singleton dimension 1.  Target sizes: [1, 3].  Tensor sizes: [1, 32]"
     ]
    }
   ],
   "source": [
    "x[0, y].view(1, -1).expand(1, x.size()[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "self.transitions: Parameter containing:\n",
      "tensor([[-2.0531,  0.1550, -0.7984, -0.7743,  1.3765],\n",
      "        [ 0.3320,  0.0276, -1.6227,  0.3256, -0.3506],\n",
      "        [-0.1191, -0.0183,  0.9328, -1.5924,  1.8704],\n",
      "        [ 0.0952, -0.8484,  0.4585, -0.6996,  0.0129],\n",
      "        [ 0.1481,  0.7436, -0.9027, -1.5715, -0.9465]], requires_grad=True)\n",
      "self.transitions ------>: Parameter containing:\n",
      "tensor([[-2.0531e+00,  1.5505e-01, -7.9837e-01, -7.7434e-01, -1.0000e+04],\n",
      "        [ 3.3201e-01,  2.7574e-02, -1.6227e+00,  3.2563e-01, -1.0000e+04],\n",
      "        [-1.1905e-01, -1.8328e-02,  9.3275e-01, -1.5924e+00, -1.0000e+04],\n",
      "        [-1.0000e+04, -1.0000e+04, -1.0000e+04, -1.0000e+04, -1.0000e+04],\n",
      "        [ 1.4810e-01,  7.4357e-01, -9.0270e-01, -1.5715e+00, -1.0000e+04]],\n",
      "       requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "transitions = nn.Parameter(torch.randn(5, 5))\n",
    "        \n",
    "print(\"self.transitions:\", transitions)\n",
    "\n",
    "# These two statements enforce the constraint that we never transfer\n",
    "# to the start tag and we never transfer from the stop tag\n",
    "transitions.data[3, :] = -10000\n",
    "transitions.data[:, 4] = -10000\n",
    "print(\"self.transitions ------>:\", transitions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-10000., -10000., -10000., -10000., -10000.]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.full((1, 5), -10000.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_out.view(len(sentence), self.hidden_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "x1 = torch.randn(10, 1, 4)\n",
    "x2 = torch.randn(2, 1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([10, 1, 4]), torch.Size([2, 1, 2]))"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1.size(), x2.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10, 4])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1.view(10, 4).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_one = torch.tensor(0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.2000)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.2000, 0.2000, 0.2000, 0.2000, 0.2000]])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_one.view(1, -1).expand(1, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
